package gods.bubble.preprocess;

import gods.bubble.common.StandardDocument;
import gods.bubble.parser.FileParser;
import gods.bubble.util.Config;
import gods.bubble.util.PathURL;
import gods.bubble.util.SimpleFileIOUtil;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Date;
import java.util.HashMap;

public class LinkExtractor {
	HashMap<String, Link> links = new HashMap<String, Link>();

	public void extractFromDir(String dataPath, String logPath)
			throws IOException {
		StringBuilder sb = new StringBuilder();
		System.out.println(new Date() + " --Start link extraction");
		File dataFile = new File(dataPath);
		File logFile = new File(logPath);

		extractFromDir(dataFile);

		for (Link link : links.values()) {
			sb.append(link.toString());
			sb.append("\n");
		}

		SimpleFileIOUtil.stringToFile(sb.toString(), logFile, "utf8");
		System.out.println(new Date() + " --Finish link extraction");
	}

	public void extractFromDir(File dataDir) throws IOException {
		File[] files = dataDir.listFiles();

		for (File file : files) {
			if (file.isDirectory()) {
				extractFromDir(file);
			} else {
				extractFromFile(file);
			}
		}
	}

	public void extractFromFile(File file) throws IOException {
		Link link = new Link();
		String url = PathURL.path2url(file.getAbsolutePath());

		link.url = url.replaceAll("\\\\", "/");

		// Check if url is a valid page

		// if it is a htm/html page then extract links inside it
		// for each inner link
		// check its anchor information & make the longest one it title
		// if it is a MSDocument/PDF then only append its filename & url
		// if others append nothing

		if (url.endsWith(".htm") || url.endsWith(".html") || url.endsWith(".jsp")) {
			StandardDocument sd = new StandardDocument();
			sd = FileParser.parser(file);

			if (sd == null) {
				return;
			}

			// For scholar search
			if (url.contains("kyybgxx.cic.tsinghua.edu.cn")) {
				String subject = sd.raw_html;

				int position = subject.indexOf("<strong>");

				if (position > 0) {
					subject = subject.substring(position + "<strong>".length());
					subject = subject
							.substring(0, subject.indexOf("</strong>"));

					link.title = subject;
				}
				
				link.url = scholarUrlConverter(link.url);
			}

			String p = sd.sublinks;

			// System.out.println(sd.sublinks);
			while (p.indexOf("<a href=\"") > 0) {
				p = p.substring(p.indexOf("<a href=\"") + 9);
				String pp = p.substring(0, p.indexOf("\""));
				String s = null;
				if (pp.startsWith("/")) {
					if (ToURL.toURL(url).indexOf("/") > 0)
						s = ToURL.toURL(url).substring(0,
								ToURL.toURL(url).indexOf("/"))
								+ pp;
				} else if (pp.startsWith("..")) {
					if (ToURL.toURL(url).lastIndexOf("/") > 0)
						s = ToURL.toURL(url).substring(0,
								ToURL.toURL(url).lastIndexOf("/"));
					if (ToURL.toURL(s).lastIndexOf("/") > 0)
						s = ToURL.toURL(s).substring(0,
								ToURL.toURL(s).lastIndexOf("/"))
								+ pp.substring(2);
				} else if (pp.startsWith(".")) {
					if (ToURL.toURL(url).lastIndexOf("/") > 0)
						if (pp.startsWith(".")) {
							pp = pp.substring(1);
						}
					s = ToURL.toURL(url).substring(0,
							ToURL.toURL(url).lastIndexOf("/"))
							+ pp;
				} else if (pp.startsWith("http") || pp.startsWith("ftp")
						|| pp.startsWith("file") || pp.startsWith("mail")
						|| pp.startsWith("www.") || pp.startsWith("mms:")) {
					s = pp;
					if (s.startsWith("http")) {
						s = s.replace("http://", "");
					}
				} else {
					if (pp.contains(".")
							&& (pp.endsWith(".doc") || pp.endsWith(".jsp")
									|| pp.endsWith("pdf")
									|| pp.endsWith("html")
									|| pp.endsWith("htm") || pp.endsWith("asp")
									|| pp.endsWith("JPG") || pp.endsWith("jpg")
									|| pp.endsWith("tsinghuaTV")
									|| pp.endsWith("swf") || pp.endsWith("xls")
									|| pp.endsWith("WMV") || pp.endsWith("avi") || pp
									.endsWith("rar"))) {
						if (ToURL.toURL(url).lastIndexOf("/") > 0)
							s = ToURL.toURL(url).substring(0,
									ToURL.toURL(url).lastIndexOf("/"))
									+ "/" + pp;
					}
				}
				if (s != null) {
					s = s.replaceAll("\\\\", "/");

					if (s.endsWith(".cn") | s.endsWith(".com")) {
						s = s + "/";
					}

					if (s.endsWith("/")) {
						s = s + "index.html";
					}
					
					if (s.contains("kyybgxx.cic.tsinghua.edu.cn")) {
						s = scholarUrlConverter(s);
					}

					if (!s.equals(link.url)) {
						link.innerLinks.add(s);
					}
				}
			}

			links.put(ToURL.toURL(url), link);
		} else {
			if (url.endsWith(".pdf") || url.endsWith(".doc")) {
				links.put(ToURL.toURL(url), link);
			}

		}
	}
	
	private String scholarUrlConverter(String scholarUrl) {
		String ret = scholarUrl;
		
		if (!ret.endsWith(".jsp"))
			return scholarUrl;
		
		ret = ret.substring(0, ret.indexOf(".jsp"));
		
		int splitPosition = ret.indexOf("detail") + "detail".length();
		
		if (splitPosition > 0) {
			String prefix = ret.substring(0, splitPosition);
			String suffix = ret.substring(splitPosition);
			
			ret = prefix + ".jsp?" + suffix;
			
			return ret;
		}
		else {
			return scholarUrl;
		}
	}

	public static void main(String args[]) throws FileNotFoundException,
			IOException {
		LinkExtractor le = new LinkExtractor();

		le.extractFromDir(Config.dataDir, Config.linkFile);
	}
}
